import Config as conf
import numpy as np
import pandas as pd
import scipy
from datetime import datetime
from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer
from sklearn.preprocessing import LabelBinarizer
from sklearn.linear_model import Ridge


# read data from train.tsv & test.tsv
print("read train data from csv")
df_train = pd.read_csv(conf.TRAIN_DATA_PATH, sep='\t')
print("read test data from csv")
df_test = pd.read_csv(conf.TEST_DATA_PATH, sep='\t')

# concat train & test together
print("concat train data & test data")
df_all = pd.concat([df_train, df_test])

# count of train data
nrow_train = df_train.shape[0]

# y_train
y_train = np.log1p(df_train['price'])

# name
# use CountVectorizer
print("name ...")
df_all['name'] = df_all['name'].fillna('none').astype('category')
count_name = CountVectorizer()
X_name = count_name.fit_transform(df_all['name'])

# item_condition_id
# use LabelBinarizer
print("item_condition_id ...")
vect_item_condition_id = LabelBinarizer(sparse_output=True)
X_item_condition_id = vect_item_condition_id.fit_transform(df_all['item_condition_id'])

# transform category_name
# use CountVectorizer
print("category_name ...")
df_all['category_name'] = df_all['category_name'].fillna('Other').astype('category')
count_category_name = CountVectorizer()
X_category_name = count_category_name.fit_transform(df_all['category_name'])

# brand_name
# use LabelBinarizer
print("brand_name ...")
df_all['brand_name'] = df_all['brand_name'].fillna('unknown').astype('category')
vect_brand = LabelBinarizer(sparse_output=True)
X_brand_name = vect_brand.fit_transform(df_all['brand_name'])

# shipping
print("shipping ...")
X_shipping = scipy.sparse.csr_matrix(df_all['shipping']).T

# item description
# use TfidfVectorizer
print("item_description ...")
df_all['item_description'] = df_all['item_description'].fillna('None')
count_description = TfidfVectorizer(stop_words='english')
X_item_description = count_description.fit_transform(df_all['item_description'])

# all
print("hstack all columns")
X_all = scipy.sparse.hstack([X_name, X_item_condition_id, X_brand_name, X_shipping, X_item_condition_id]).tocsr()

# train
print("model training...")
X_train = X_all[:nrow_train]
model = Ridge(solver='lsqr', fit_intercept=False)
model.fit(X_train, y_train)

# test
print("model predicting...")
X_test = X_all[nrow_train:]
preds = model.predict(X_test)

print("save preds to csv file")
df_test['price'] = np.expm1(preds)
df_test[['test_id', 'price']].to_csv(conf.RESULT_CSV_DIRECTORY_PATH + str(datetime.now()) + '.csv', index=False)
print("complete!!!!")